by ARYAN JAIN
Context: Company X intends to build a face identification model to recognise human faces.
Data Description:
The dataset comprises of images and its mask where there is a human face
Domain: Face recognition
Objectives: Face Aligned Face Dataset from Pinterest. This dataset contains 10,770 images for 100 people. All images are taken from 'Pinterest' and aligned using dlib library
Key Tasks: Use a pre-trained model trained on Face recognition to recognise similar faces. The intent is to recognise whether two given faces are of the same person or not.
- Load the dataset and create the metadata.
- Check some samples of metadata.
- Load the pre-trained model and weights.
- Generate Embedding vectors for each face in the dataset.
- Build distance metrics for identifying the distance between two given images.
- Use PCA for dimensionality reduction.
- Build an SVM classifier in order to map each image to its right person.
- Import the the test image. Display the image. Use the SVM trained model to predict the face.
# Data Management and Vizualization
import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
# Deep Learning
import cv2
from skimage import io,transform
from tqdm.notebook import tqdm
from tensorflow.keras.layers import ZeroPadding2D, Convolution2D, MaxPooling2D, Dropout, Flatten, Activation
from tensorflow.python.keras.preprocessing.image import image, load_img, ImageDataGenerator
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.utils import to_categorical
from tensorflow.keras import optimizers
from mpl_toolkits.axes_grid1 import ImageGrid
import tensorflow as tf
# Standard ML
from sklearn.svm import SVC
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.decomposition import PCA
# Python Imaging Library: to perform operations on images
import PIL
from PIL import Image
# warnings
import warnings
warnings.simplefilter("ignore")
# Define dataset folder & reusable variables
# Source data directory / Training dataset
# The PINS folder contains one sub-folder for each class which contains images of the respective class
data_dir = 'PINS'
# Function to investigate classes
def get_classes(data_dir):
class_names = {}
for i,class_name in enumerate(sorted(os.listdir(data_dir))):
class_names.setdefault(i,class_name)
return class_names
# Let's look at what classes exist
list_of_classes = get_classes (data_dir)
list_of_classes
{0: 'pins_Aaron Paul',
1: 'pins_Alvaro Morte',
2: 'pins_Amanda Crew',
3: 'pins_Amaury Nolasco',
4: 'pins_Anna Gunn',
5: 'pins_Benedict Cumberbatch',
6: 'pins_Betsy Brandt',
7: 'pins_Brenton Thwaites',
8: 'pins_Brit Marling',
9: 'pins_Bryan Cranston',
10: 'pins_Caity Lotz',
11: 'pins_Cameron Monaghan',
12: 'pins_Chance Perdomo',
13: 'pins_Chris Evans',
14: 'pins_Chris Pratt',
15: 'pins_Cobie Smulders',
16: 'pins_Danielle Panabaker',
17: 'pins_Dave Franco',
18: 'pins_Dominic Purcell',
19: 'pins_Dwayne Johnson',
20: 'pins_Emilia Clarke',
21: 'pins_Emily Bett Rickards',
22: 'pins_Emma Stone',
23: 'pins_Gwyneth Paltrow',
24: 'pins_Henry Cavil',
25: 'pins_Jason Momoa',
26: 'pins_Jeremy Renner',
27: 'pins_Jesse Eisenberg',
28: 'pins_Jim Parsons',
29: 'pins_Jon Bernthal',
30: 'pins_Josh Radnor',
31: 'pins_Kit Harington',
32: 'pins_Krysten Ritter',
33: 'pins_Kumail Nanjiani',
34: 'pins_Maisie Williams',
35: 'pins_Mark Ruffalo',
36: 'pins_Martin Starr',
37: 'pins_Melissa benoit',
38: 'pins_Mike Colter',
39: 'pins_Morena Baccarin',
40: 'pins_Morgan Freeman',
41: 'pins_Natalie Portman ',
42: 'pins_Neil Patrick Harris',
43: 'pins_Paul Rudd',
44: 'pins_Pedro Alonso',
45: 'pins_Peter Dinklage',
46: 'pins_RJ Mitte',
47: 'pins_Rami Melek',
48: 'pins_Robert Knepper',
49: 'pins_Robin Taylor',
50: 'pins_Ryan Reynolds',
51: 'pins_Sarah Wayne Callies',
52: 'pins_Scarlett Johansson',
53: 'pins_Sebastian Stan',
54: 'pins_Sophie Turner',
55: 'pins_Stephen Amell',
56: 'pins_Sundar Pichai',
57: 'pins_Thomas Middleditch',
58: 'pins_Tom Cavanagh',
59: 'pins_Ursula Corbero',
60: 'pins_Wentworth Miller',
61: 'pins_Willa Holland',
62: 'pins_William Fichtner',
63: 'pins_alexandra daddario',
64: 'pins_alycia debnam carey face',
65: 'pins_amber heard face',
66: 'pins_anne hathaway',
67: 'pins_barbara palvin face',
68: 'pins_bellamy blake face',
69: 'pins_bill gates',
70: 'pins_brie larson',
71: 'pins_chadwick boseman face',
72: 'pins_david mazouz',
73: 'pins_drake',
74: 'pins_dua lipa face',
75: 'pins_eliza taylor',
76: 'pins_elizabeth olsen face',
77: 'pins_elon musk',
78: 'pins_emma watson face',
79: 'pins_gal gadot face',
80: 'pins_grant gustin face',
81: 'pins_jason isaacs',
82: 'pins_jeff bezos',
83: 'pins_kiernan shipka ',
84: 'pins_kristen stewart face',
85: 'pins_lindsey morgan face',
86: 'pins_margot robbie face',
87: 'pins_maria pedraza',
88: 'pins_mark zuckerberg',
89: 'pins_miguel herran',
90: 'pins_millie bobby brown',
91: 'pins_rihanna',
92: 'pins_robert downey jr face',
93: 'pins_sean pertwee',
94: 'pins_selena gomez',
95: 'pins_shakira',
96: 'pins_tati gabrielle',
97: 'pins_taylor swift',
98: 'pins_tom holland face',
99: 'pins_zendaya'}
# Just for fun:
# Let's create dictionaries with names as keys and class as value and vice versa
dirs = sorted(os.listdir(data_dir))
#Build the Name to Class ID dictionary
name_to_classid = {d.replace('pins_','').replace('face',''):i for i,d in enumerate(dirs)}
print (name_to_classid)
print ('\n\n')
#Build Class ID to Name dictionary
classid_to_name = dict((y,x) for x,y in name_to_classid.items())
print (classid_to_name)
print(f'\nNumber of classes : {str(len(name_to_classid))}')
{'Aaron Paul': 0, 'Alvaro Morte': 1, 'Amanda Crew': 2, 'Amaury Nolasco': 3, 'Anna Gunn': 4, 'Benedict Cumberbatch': 5, 'Betsy Brandt': 6, 'Brenton Thwaites': 7, 'Brit Marling': 8, 'Bryan Cranston': 9, 'Caity Lotz': 10, 'Cameron Monaghan': 11, 'Chance Perdomo': 12, 'Chris Evans': 13, 'Chris Pratt': 14, 'Cobie Smulders': 15, 'Danielle Panabaker': 16, 'Dave Franco': 17, 'Dominic Purcell': 18, 'Dwayne Johnson': 19, 'Emilia Clarke': 20, 'Emily Bett Rickards': 21, 'Emma Stone': 22, 'Gwyneth Paltrow': 23, 'Henry Cavil': 24, 'Jason Momoa': 25, 'Jeremy Renner': 26, 'Jesse Eisenberg': 27, 'Jim Parsons': 28, 'Jon Bernthal': 29, 'Josh Radnor': 30, 'Kit Harington': 31, 'Krysten Ritter': 32, 'Kumail Nanjiani': 33, 'Maisie Williams': 34, 'Mark Ruffalo': 35, 'Martin Starr': 36, 'Melissa benoit': 37, 'Mike Colter': 38, 'Morena Baccarin': 39, 'Morgan Freeman': 40, 'Natalie Portman ': 41, 'Neil Patrick Harris': 42, 'Paul Rudd': 43, 'Pedro Alonso': 44, 'Peter Dinklage': 45, 'RJ Mitte': 46, 'Rami Melek': 47, 'Robert Knepper': 48, 'Robin Taylor': 49, 'Ryan Reynolds': 50, 'Sarah Wayne Callies': 51, 'Scarlett Johansson': 52, 'Sebastian Stan': 53, 'Sophie Turner': 54, 'Stephen Amell': 55, 'Sundar Pichai': 56, 'Thomas Middleditch': 57, 'Tom Cavanagh': 58, 'Ursula Corbero': 59, 'Wentworth Miller': 60, 'Willa Holland': 61, 'William Fichtner': 62, 'alexandra daddario': 63, 'alycia debnam carey ': 64, 'amber heard ': 65, 'anne hathaway': 66, 'barbara palvin ': 67, 'bellamy blake ': 68, 'bill gates': 69, 'brie larson': 70, 'chadwick boseman ': 71, 'david mazouz': 72, 'drake': 73, 'dua lipa ': 74, 'eliza taylor': 75, 'elizabeth olsen ': 76, 'elon musk': 77, 'emma watson ': 78, 'gal gadot ': 79, 'grant gustin ': 80, 'jason isaacs': 81, 'jeff bezos': 82, 'kiernan shipka ': 83, 'kristen stewart ': 84, 'lindsey morgan ': 85, 'margot robbie ': 86, 'maria pedraza': 87, 'mark zuckerberg': 88, 'miguel herran': 89, 'millie bobby brown': 90, 'rihanna': 91, 'robert downey jr ': 92, 'sean pertwee': 93, 'selena gomez': 94, 'shakira': 95, 'tati gabrielle': 96, 'taylor swift': 97, 'tom holland ': 98, 'zendaya': 99}
{0: 'Aaron Paul', 1: 'Alvaro Morte', 2: 'Amanda Crew', 3: 'Amaury Nolasco', 4: 'Anna Gunn', 5: 'Benedict Cumberbatch', 6: 'Betsy Brandt', 7: 'Brenton Thwaites', 8: 'Brit Marling', 9: 'Bryan Cranston', 10: 'Caity Lotz', 11: 'Cameron Monaghan', 12: 'Chance Perdomo', 13: 'Chris Evans', 14: 'Chris Pratt', 15: 'Cobie Smulders', 16: 'Danielle Panabaker', 17: 'Dave Franco', 18: 'Dominic Purcell', 19: 'Dwayne Johnson', 20: 'Emilia Clarke', 21: 'Emily Bett Rickards', 22: 'Emma Stone', 23: 'Gwyneth Paltrow', 24: 'Henry Cavil', 25: 'Jason Momoa', 26: 'Jeremy Renner', 27: 'Jesse Eisenberg', 28: 'Jim Parsons', 29: 'Jon Bernthal', 30: 'Josh Radnor', 31: 'Kit Harington', 32: 'Krysten Ritter', 33: 'Kumail Nanjiani', 34: 'Maisie Williams', 35: 'Mark Ruffalo', 36: 'Martin Starr', 37: 'Melissa benoit', 38: 'Mike Colter', 39: 'Morena Baccarin', 40: 'Morgan Freeman', 41: 'Natalie Portman ', 42: 'Neil Patrick Harris', 43: 'Paul Rudd', 44: 'Pedro Alonso', 45: 'Peter Dinklage', 46: 'RJ Mitte', 47: 'Rami Melek', 48: 'Robert Knepper', 49: 'Robin Taylor', 50: 'Ryan Reynolds', 51: 'Sarah Wayne Callies', 52: 'Scarlett Johansson', 53: 'Sebastian Stan', 54: 'Sophie Turner', 55: 'Stephen Amell', 56: 'Sundar Pichai', 57: 'Thomas Middleditch', 58: 'Tom Cavanagh', 59: 'Ursula Corbero', 60: 'Wentworth Miller', 61: 'Willa Holland', 62: 'William Fichtner', 63: 'alexandra daddario', 64: 'alycia debnam carey ', 65: 'amber heard ', 66: 'anne hathaway', 67: 'barbara palvin ', 68: 'bellamy blake ', 69: 'bill gates', 70: 'brie larson', 71: 'chadwick boseman ', 72: 'david mazouz', 73: 'drake', 74: 'dua lipa ', 75: 'eliza taylor', 76: 'elizabeth olsen ', 77: 'elon musk', 78: 'emma watson ', 79: 'gal gadot ', 80: 'grant gustin ', 81: 'jason isaacs', 82: 'jeff bezos', 83: 'kiernan shipka ', 84: 'kristen stewart ', 85: 'lindsey morgan ', 86: 'margot robbie ', 87: 'maria pedraza', 88: 'mark zuckerberg', 89: 'miguel herran', 90: 'millie bobby brown', 91: 'rihanna', 92: 'robert downey jr ', 93: 'sean pertwee', 94: 'selena gomez', 95: 'shakira', 96: 'tati gabrielle', 97: 'taylor swift', 98: 'tom holland ', 99: 'zendaya'}
Number of classes : 100
# Based on above, the dataset contains 100 classes (well known people)
# Creating a dataframe to collect Metadata: file_nm, class_nm, class_id
# What was the error Not a directory: 'PINS/.DS_Store. I had to delete this file manually
metadata = []
for directory in os.listdir(data_dir):
class_name = directory.replace('pins_','').replace('face','')
class_id = name_to_classid[class_name]
for file in os.listdir(os.path.join(data_dir, directory)):
file_name = os.path.join(os.path.join(data_dir, directory), file)
metadata.append([file_name, class_name, class_id])
#data = pd.DataFrame(metadata, columns=['File', 'Celeb-Name','ClassID'])
df = pd.DataFrame(metadata, columns=['file_nm', 'class_nm','class_id'])
df.head()
| file_nm | class_nm | class_id | |
|---|---|---|---|
| 0 | PINS/pins_mark zuckerberg/mark zuckerberg1.jpg | mark zuckerberg | 88 |
| 1 | PINS/pins_mark zuckerberg/mark zuckerberg50.jpg | mark zuckerberg | 88 |
| 2 | PINS/pins_mark zuckerberg/mark zuckerberg44.jpg | mark zuckerberg | 88 |
| 3 | PINS/pins_mark zuckerberg/mark zuckerberg45.jpg | mark zuckerberg | 88 |
| 4 | PINS/pins_mark zuckerberg/mark zuckerberg51.jpg | mark zuckerberg | 88 |
# Let's print number of images and classes
print ('No of Images = ', df.shape[0])
print ('No of Classes = ', len(list_of_classes))
No of Images = 10770 No of Classes = 100
# Let's check how many pictures of each person we have
pd.set_option("max_rows", None) # display all rows
df.pivot_table(index=['class_id', 'class_nm'], values='file_nm',aggfunc=len) # pivot table
| file_nm | ||
|---|---|---|
| class_id | class_nm | |
| 0 | Aaron Paul | 86 |
| 1 | Alvaro Morte | 91 |
| 2 | Amanda Crew | 118 |
| 3 | Amaury Nolasco | 113 |
| 4 | Anna Gunn | 66 |
| 5 | Benedict Cumberbatch | 81 |
| 6 | Betsy Brandt | 68 |
| 7 | Brenton Thwaites | 130 |
| 8 | Brit Marling | 122 |
| 9 | Bryan Cranston | 80 |
| 10 | Caity Lotz | 123 |
| 11 | Cameron Monaghan | 112 |
| 12 | Chance Perdomo | 86 |
| 13 | Chris Evans | 88 |
| 14 | Chris Pratt | 141 |
| 15 | Cobie Smulders | 130 |
| 16 | Danielle Panabaker | 115 |
| 17 | Dave Franco | 126 |
| 18 | Dominic Purcell | 107 |
| 19 | Dwayne Johnson | 124 |
| 20 | Emilia Clarke | 154 |
| 21 | Emily Bett Rickards | 76 |
| 22 | Emma Stone | 128 |
| 23 | Gwyneth Paltrow | 121 |
| 24 | Henry Cavil | 134 |
| 25 | Jason Momoa | 133 |
| 26 | Jeremy Renner | 119 |
| 27 | Jesse Eisenberg | 93 |
| 28 | Jim Parsons | 109 |
| 29 | Jon Bernthal | 61 |
| 30 | Josh Radnor | 101 |
| 31 | Kit Harington | 105 |
| 32 | Krysten Ritter | 115 |
| 33 | Kumail Nanjiani | 90 |
| 34 | Maisie Williams | 148 |
| 35 | Mark Ruffalo | 118 |
| 36 | Martin Starr | 48 |
| 37 | Melissa benoit | 122 |
| 38 | Mike Colter | 71 |
| 39 | Morena Baccarin | 132 |
| 40 | Morgan Freeman | 97 |
| 41 | Natalie Portman | 117 |
| 42 | Neil Patrick Harris | 73 |
| 43 | Paul Rudd | 127 |
| 44 | Pedro Alonso | 77 |
| 45 | Peter Dinklage | 94 |
| 46 | RJ Mitte | 71 |
| 47 | Rami Melek | 75 |
| 48 | Robert Knepper | 95 |
| 49 | Robin Taylor | 99 |
| 50 | Ryan Reynolds | 97 |
| 51 | Sarah Wayne Callies | 120 |
| 52 | Scarlett Johansson | 146 |
| 53 | Sebastian Stan | 107 |
| 54 | Sophie Turner | 121 |
| 55 | Stephen Amell | 100 |
| 56 | Sundar Pichai | 89 |
| 57 | Thomas Middleditch | 82 |
| 58 | Tom Cavanagh | 100 |
| 59 | Ursula Corbero | 80 |
| 60 | Wentworth Miller | 113 |
| 61 | Willa Holland | 147 |
| 62 | William Fichtner | 139 |
| 63 | alexandra daddario | 165 |
| 64 | alycia debnam carey | 144 |
| 65 | amber heard | 151 |
| 66 | anne hathaway | 151 |
| 67 | barbara palvin | 142 |
| 68 | bellamy blake | 89 |
| 69 | bill gates | 86 |
| 70 | brie larson | 128 |
| 71 | chadwick boseman | 119 |
| 72 | david mazouz | 104 |
| 73 | drake | 38 |
| 74 | dua lipa | 137 |
| 75 | eliza taylor | 105 |
| 76 | elizabeth olsen | 181 |
| 77 | elon musk | 85 |
| 78 | emma watson | 163 |
| 79 | gal gadot | 158 |
| 80 | grant gustin | 122 |
| 81 | jason isaacs | 125 |
| 82 | jeff bezos | 88 |
| 83 | kiernan shipka | 167 |
| 84 | kristen stewart | 118 |
| 85 | lindsey morgan | 76 |
| 86 | margot robbie | 140 |
| 87 | maria pedraza | 68 |
| 88 | mark zuckerberg | 62 |
| 89 | miguel herran | 81 |
| 90 | millie bobby brown | 82 |
| 91 | rihanna | 120 |
| 92 | robert downey jr | 107 |
| 93 | sean pertwee | 82 |
| 94 | selena gomez | 93 |
| 95 | shakira | 50 |
| 96 | tati gabrielle | 65 |
| 97 | taylor swift | 99 |
| 98 | tom holland | 119 |
| 99 | zendaya | 109 |
# Let's visualize the class distribution using countplot
sns.set(rc={'figure.figsize':(30,10)})
sns.countplot(df['class_id']) ;
# Just for fun & learning, using another method to display random images
# Function to display 5 images of each class
def show_sample_images(class_idx):
i = 0
plt.figure()
name = df[df['class_id'] == class_idx]['file_nm'].values[-1]
celeb_name = name.split("/")[1].split("_")[1]
print(f"Images of {celeb_name.title()}")
for filepath in df[df['class_id'] == class_idx]['file_nm'].values[:5]:
i=i+1
img=io.imread(filepath)
plt.subplot(1,5,i)
plt.imshow(img)
# Choose 5 random classes, call function to display 5 random images of each class
classes = [10, 56, 79, 82, 99]
for id in classes:
show_sample_images(id)
Images of Caity Lotz Images of Sundar Pichai Images of Gal Gadot Face Images of Jeff Bezos Images of Zendaya
# Deep Learning
Below is the VGG Face model (reference source >>> internet )
# VGG Model
# Loading the pre-trained model and weights
# The problem statement includes the weights file vgg_face_weights.h5
# Next step are to build the VGG model, and load the weights from the given weights file
# Build the VGG Model
vgg_model = Sequential()
vgg_model.add(ZeroPadding2D((1, 1), input_shape = (224, 224, 3)))
vgg_model.add(Convolution2D(64, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(64, (3, 3), activation = 'relu'))
vgg_model.add(MaxPooling2D((2, 2), strides = (2, 2)))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(128, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(128, (3, 3), activation = 'relu'))
vgg_model.add(MaxPooling2D((2, 2), strides = (2, 2)))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(256, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(256, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(256, (3, 3), activation = 'relu'))
vgg_model.add(MaxPooling2D((2, 2), strides = (2, 2)))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(512, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(512, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(512, (3, 3), activation = 'relu'))
vgg_model.add(MaxPooling2D((2, 2), strides =(2, 2)))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(512, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(512, (3, 3), activation = 'relu'))
vgg_model.add(ZeroPadding2D((1, 1)))
vgg_model.add(Convolution2D(512, (3, 3), activation = 'relu'))
vgg_model.add(MaxPooling2D((2, 2), strides=(2, 2)))
vgg_model.add(Convolution2D(4096, (7, 7), activation = 'relu'))
vgg_model.add(Dropout(0.5))
vgg_model.add(Convolution2D(4096, (1, 1), activation = 'relu'))
vgg_model.add(Dropout(0.5))
vgg_model.add(Convolution2D(2622, (1, 1)))
vgg_model.add(Flatten())
vgg_model.add(Activation('softmax'))
# Load provided weight file & display summary
vgg_model.load_weights('Part 3 - vgg_face_weights.h5')
vgg_model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= zero_padding2d (ZeroPadding2 (None, 226, 226, 3) 0 _________________________________________________________________ conv2d (Conv2D) (None, 224, 224, 64) 1792 _________________________________________________________________ zero_padding2d_1 (ZeroPaddin (None, 226, 226, 64) 0 _________________________________________________________________ conv2d_1 (Conv2D) (None, 224, 224, 64) 36928 _________________________________________________________________ max_pooling2d (MaxPooling2D) (None, 112, 112, 64) 0 _________________________________________________________________ zero_padding2d_2 (ZeroPaddin (None, 114, 114, 64) 0 _________________________________________________________________ conv2d_2 (Conv2D) (None, 112, 112, 128) 73856 _________________________________________________________________ zero_padding2d_3 (ZeroPaddin (None, 114, 114, 128) 0 _________________________________________________________________ conv2d_3 (Conv2D) (None, 112, 112, 128) 147584 _________________________________________________________________ max_pooling2d_1 (MaxPooling2 (None, 56, 56, 128) 0 _________________________________________________________________ zero_padding2d_4 (ZeroPaddin (None, 58, 58, 128) 0 _________________________________________________________________ conv2d_4 (Conv2D) (None, 56, 56, 256) 295168 _________________________________________________________________ zero_padding2d_5 (ZeroPaddin (None, 58, 58, 256) 0 _________________________________________________________________ conv2d_5 (Conv2D) (None, 56, 56, 256) 590080 _________________________________________________________________ zero_padding2d_6 (ZeroPaddin (None, 58, 58, 256) 0 _________________________________________________________________ conv2d_6 (Conv2D) (None, 56, 56, 256) 590080 _________________________________________________________________ max_pooling2d_2 (MaxPooling2 (None, 28, 28, 256) 0 _________________________________________________________________ zero_padding2d_7 (ZeroPaddin (None, 30, 30, 256) 0 _________________________________________________________________ conv2d_7 (Conv2D) (None, 28, 28, 512) 1180160 _________________________________________________________________ zero_padding2d_8 (ZeroPaddin (None, 30, 30, 512) 0 _________________________________________________________________ conv2d_8 (Conv2D) (None, 28, 28, 512) 2359808 _________________________________________________________________ zero_padding2d_9 (ZeroPaddin (None, 30, 30, 512) 0 _________________________________________________________________ conv2d_9 (Conv2D) (None, 28, 28, 512) 2359808 _________________________________________________________________ max_pooling2d_3 (MaxPooling2 (None, 14, 14, 512) 0 _________________________________________________________________ zero_padding2d_10 (ZeroPaddi (None, 16, 16, 512) 0 _________________________________________________________________ conv2d_10 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ zero_padding2d_11 (ZeroPaddi (None, 16, 16, 512) 0 _________________________________________________________________ conv2d_11 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ zero_padding2d_12 (ZeroPaddi (None, 16, 16, 512) 0 _________________________________________________________________ conv2d_12 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ max_pooling2d_4 (MaxPooling2 (None, 7, 7, 512) 0 _________________________________________________________________ conv2d_13 (Conv2D) (None, 1, 1, 4096) 102764544 _________________________________________________________________ dropout (Dropout) (None, 1, 1, 4096) 0 _________________________________________________________________ conv2d_14 (Conv2D) (None, 1, 1, 4096) 16781312 _________________________________________________________________ dropout_1 (Dropout) (None, 1, 1, 4096) 0 _________________________________________________________________ conv2d_15 (Conv2D) (None, 1, 1, 2622) 10742334 _________________________________________________________________ flatten (Flatten) (None, 2622) 0 _________________________________________________________________ activation (Activation) (None, 2622) 0 ================================================================= Total params: 145,002,878 Trainable params: 145,002,878 Non-trainable params: 0 _________________________________________________________________
# What we have done & next steps:
# We have already loaded the pre-trained weights for the VGG-16 network
# No need train it further
# Will create embedding vectors for facial verification before Flatten layer
# Use the embedding vectors (face descriptors) to calculate the distance between images
# Get the face descriptor
# Creating a model that takes the input, but then takes output of Flatten layer, rather than softmax
face_descriptor = Model(inputs = vgg_model.layers[0].input, outputs = vgg_model.layers[-2].output)
# Using face_descriptor.predict, I can get embedding vectors for each image
# Generate Embedding vectors for each face in the dataset
def read_image(path):
img = cv2.imread(path, 1)
# OpenCV loads images with color channels in BGR order. So we need to reverse them
return img[...,::-1]
# Generate Embedding Vector for first image, validate the the shape
img_path = df['file_nm'][0]
img = read_image(img_path)
# Normalize the input and convert to type float
img = (img / 255.).astype(np.float32)
img = cv2.resize(img, dsize = (224, 224))
print('Shape of image : ', img.shape)
# Obtain the embedding vector for the above image using face_descriptor model and print the shape
embedding_vector = face_descriptor.predict(np.expand_dims(img, axis = 0))[0]
print('Shape of Embedding Vector : ', embedding_vector.shape)
Shape of image : (224, 224, 3) Shape of Embedding Vector : (2622,)
# The length of the Embedding Vector is 2622.
# Next Step: generate the Embedding Vector for each image in the dataset
# Note: the number of embeddings should match number of images in the dataset
embeddings = []
for index, filename in tqdm(enumerate(df.iloc[:, 0])):
img = read_image(filename)
img = (img / 255.).astype(np.float32)
img = cv2.resize(img, dsize = (224, 224))
embeddings.append(face_descriptor.predict(np.expand_dims(img, axis = 0))[0])
print ('Total number of embeddings : ', len(embeddings))
Total number of embeddings : 10770
# OBSERVATION: As expected, total number of embeddings match the total number of images in the dataset
# NEXT STEPS:
# Write a function to calculate the distance between 2 images
# Display the distance
$ d(p,q) = \sqrt{\sum \limits_{i=1}^{n} (q_{i} - p_{i}) ^2} $
p,q = two points in Euclidean n-space
$q_{i}, p_{i} $ = Euclidean vectors, starting from the origin of the space (initial point)
n = n-space
# function that measure the Euclidean Distance between two points
def euc_dist(vec1, vec2):
return np.sqrt(np.sum((vec1-vec2)**2))
# Display the distance between given pair of images
def check_pair(idx1, idx2):
plt.figure(figsize = (8, 3))
plt.suptitle(f'Distance = {euc_dist(embeddings[idx1], embeddings[idx2]):.2f}')
plt.subplot(121)
img1 = df['file_nm'].iloc[idx1]
print ('Image 1 : ', img1)
plt.imshow(read_image(img1))
plt.subplot(122)
img2 = df['file_nm'].iloc[idx2]
print ('Image 2 : ', img2, '\n')
plt.imshow(read_image(img2))
# Let's pick one anchor image and measure distance from 5 random images
check_pair(5, 5)
check_pair(5, 8)
check_pair(5, 300)
check_pair(5, 500)
Image 1 : PINS/pins_mark zuckerberg/mark zuckerberg0.jpg Image 2 : PINS/pins_mark zuckerberg/mark zuckerberg0.jpg Image 1 : PINS/pins_mark zuckerberg/mark zuckerberg0.jpg Image 2 : PINS/pins_mark zuckerberg/mark zuckerberg53.jpg Image 1 : PINS/pins_mark zuckerberg/mark zuckerberg0.jpg Image 2 : PINS/pins_barbara palvin face/barbara palvin face5.jpg Image 1 : PINS/pins_mark zuckerberg/mark zuckerberg0.jpg Image 2 : PINS/pins_Jason Momoa/Jason Momoa76.jpg
# OBSERVATIONS:
# 1st pair: exact same image, the distance = 0
# 2nd pair: same person but different position, angle, etc. As a result, the distance increased 0.52
# 3rd pair: different people of different genders so the distance increased significantly 0.80
# 4th pair: different people of the same gender so the distance dropped slightly 0.64
Split the dataset into training and validation
# setting values for variables
random_s = 7
test_s = 0.3
# The embeddings list has the embeddings of each of the 10770 images
# We get the corresponding label from the classID column of data dataframe
y = df['class_id']
print ('Number of embeddings : ', len(embeddings))
print ('Number of Labels : ', len(y))
# Split the data into Train and Test dataset
X_train, X_test, y_train, y_test = train_test_split(np.array(embeddings), y, test_size=test_s, random_state=random_s)
X_train.shape, y_train.shape, X_test.shape, y_test.shape
Number of embeddings : 10770 Number of Labels : 10770
((7539, 2622), (7539,), (3231, 2622), (3231,))
# Let's standarize the features, we could skip this step if we applied BatchNorm in the network
sc = StandardScaler()
Xsc_train = sc.fit_transform(X_train)
Xsc_test = sc.transform(X_test)
# PCA for dimention reduction
# Step 1 - Create covariance matrix
cov_mat = np.cov(Xsc_train.T)
# Step 2- Get eigen values and eigen vector
eig_vals, eig_vecs = np.linalg.eig(cov_mat)
print('Eigen Vectors \n%s', eig_vecs)
print('\n Eigen Values \n%s', eig_vals)
Eigen Vectors %s [[-2.36873712e-02 -2.07472871e-02 -1.80319214e-04 ... -1.90094287e-02 -4.05639181e-03 4.15639492e-04] [ 8.94490000e-03 -5.85544830e-05 1.92279050e-03 ... -9.41898269e-03 1.31071731e-02 3.51803420e-02] [-3.28646674e-03 -8.51692683e-03 9.72900806e-03 ... 1.60232482e-02 -2.36167037e-03 -5.40416113e-03] ... [-7.77369216e-03 2.36262104e-02 -1.08441120e-02 ... 8.24551477e-03 -2.81712565e-03 3.63036930e-02] [ 2.18229537e-02 4.87502985e-03 3.35335432e-03 ... -2.15401891e-02 8.02249909e-03 -9.97934792e-03] [ 3.87195261e-02 -1.08608302e-02 1.82938404e-02 ... 2.91680952e-02 -2.15596994e-02 -2.57737612e-02]] Eigen Values %s [3.58336835e+02 1.41260544e+02 1.03392577e+02 ... 1.30326771e-03 1.30686197e-03 1.30589496e-03]
# Display cumulative variance explained
tot_eig_val = sum(eig_vals)
var_exp = [(i /tot_eig_val) * 100 for i in sorted(eig_vals, reverse = True)]
cum_var_exp = np.cumsum(var_exp)
print('Cumulative Variance Explained : ', cum_var_exp)
Cumulative Variance Explained : [ 13.66473318 19.05152975 22.99427814 ... 99.99999985 99.99999999 100. ]
# We want the components that explain 95% of the variance, so let's set a threshold and see how many components explain it
thres = 95
res = list(filter(lambda i: i > thres, cum_var_exp))[0]
index = (cum_var_exp.tolist().index(res))
print(f'Index of element just greater than {thres} % : {str(index)}')
Index of element just greater than 95 % : 342
# the first 345 elements explain 95% variance and we will use this as our threshold
# Let's visualize number of components with variance explained to confirm this
plt.figure(figsize = (12 , 5))
plt.bar(range(1, eig_vals.size + 1), var_exp, alpha = 0.5, align = 'center')
plt.step(range(1, eig_vals.size + 1), cum_var_exp, where = 'mid', label = 'Cumulative explained variance')
plt.axhline(y = thres, color = 'g', linestyle = '--')
plt.axvline(x = index, color = 'g', linestyle = '--')
plt.ylabel('Explained Variance Ratio')
plt.xlabel('Principal Components')
plt.legend(loc = 'best')
plt.tight_layout()
plt.show()
# # Alternate simpler method
# plt.plot(cum_var_exp)
# Reducing the dimensions
# we are generating only 8 PCA dimensions (dimensionality reduction from 18 to 8)
## We can manually code in 345 elements, or just input 0.95 and it will automatically take how many components
## in order to explain 95% of the variance
# pca = PCA(n_components=345)
pca = PCA(n_components = 0.95, random_state = random_s, svd_solver = 'full', whiten = True)
pca.fit(Xsc_train)
X_train_pca = pca.transform(Xsc_train)
X_test_pca = pca.transform(Xsc_test)
display(X_train_pca.shape, X_test_pca.shape)
# NOTE - we are generating only 8 PCA dimensions (dimensionality reduction from 18 to 8)
(7539, 343)
(3231, 343)
# NEXT STEP
# build SVM classifier to predict Class Id
# We can hypertune with GridSearchCV, but I will use the parameters that seem to perform the great in most cases
svc_model = SVC(C = 1, gamma = 0.001, kernel = 'rbf', class_weight = 'balanced')
svc_model.fit(X_train_pca, y_train)
print('Model Accuracy for train set : {0:.3f}'.format(svc_model.score(X_train_pca, y_train)))
Model Accuracy for train set : 0.994
# Predict
y_pred = svc_model.predict(X_test_pca)
print('Model Accuracy for test set : {0:.3f}'.format(accuracy_score(y_test, y_pred).round(3)))
Model Accuracy for test set : 0.962
# MODEL PERFORMACE
# The SVM classifier gives an accuracy of 96.3% on test data. This is pretty good.
# Let's print the Classification Report
print('Classification Report: \n{}'.format(classification_report(y_test, y_pred,
target_names = [y for x,y in classid_to_name.items()])))
Classification Report:
precision recall f1-score support
Aaron Paul 1.00 0.92 0.96 24
Alvaro Morte 0.87 1.00 0.93 27
Amanda Crew 0.97 0.97 0.97 29
Amaury Nolasco 1.00 1.00 1.00 31
Anna Gunn 1.00 1.00 1.00 24
Benedict Cumberbatch 1.00 0.91 0.95 23
Betsy Brandt 1.00 1.00 1.00 20
Brenton Thwaites 0.95 0.92 0.94 39
Brit Marling 1.00 0.94 0.97 35
Bryan Cranston 0.92 1.00 0.96 22
Caity Lotz 0.94 0.97 0.95 30
Cameron Monaghan 0.95 1.00 0.97 39
Chance Perdomo 1.00 1.00 1.00 31
Chris Evans 0.92 0.89 0.91 27
Chris Pratt 1.00 0.98 0.99 41
Cobie Smulders 1.00 1.00 1.00 41
Danielle Panabaker 1.00 0.94 0.97 33
Dave Franco 0.92 0.95 0.94 38
Dominic Purcell 1.00 0.94 0.97 34
Dwayne Johnson 0.97 1.00 0.98 30
Emilia Clarke 0.91 0.98 0.94 43
Emily Bett Rickards 0.81 0.96 0.88 23
Emma Stone 1.00 0.95 0.97 41
Gwyneth Paltrow 0.97 1.00 0.99 39
Henry Cavil 0.97 0.97 0.97 36
Jason Momoa 0.95 1.00 0.97 39
Jeremy Renner 1.00 1.00 1.00 34
Jesse Eisenberg 0.84 0.96 0.90 27
Jim Parsons 1.00 1.00 1.00 36
Jon Bernthal 0.95 0.95 0.95 19
Josh Radnor 1.00 1.00 1.00 29
Kit Harington 0.97 0.97 0.97 30
Krysten Ritter 1.00 0.94 0.97 35
Kumail Nanjiani 1.00 0.97 0.98 29
Maisie Williams 0.98 0.92 0.95 52
Mark Ruffalo 0.97 0.95 0.96 40
Martin Starr 1.00 1.00 1.00 14
Melissa benoit 0.96 1.00 0.98 44
Mike Colter 1.00 0.95 0.98 22
Morena Baccarin 0.97 0.95 0.96 37
Morgan Freeman 1.00 0.97 0.99 36
Natalie Portman 0.97 0.90 0.93 39
Neil Patrick Harris 1.00 0.97 0.98 30
Paul Rudd 1.00 0.94 0.97 32
Pedro Alonso 0.76 0.96 0.85 23
Peter Dinklage 0.93 1.00 0.96 25
RJ Mitte 0.94 0.94 0.94 16
Rami Melek 1.00 0.96 0.98 24
Robert Knepper 1.00 0.97 0.99 39
Robin Taylor 0.97 0.97 0.97 37
Ryan Reynolds 1.00 1.00 1.00 31
Sarah Wayne Callies 0.97 0.86 0.91 42
Scarlett Johansson 0.97 0.88 0.92 34
Sebastian Stan 1.00 1.00 1.00 28
Sophie Turner 0.97 0.95 0.96 39
Stephen Amell 0.97 0.90 0.93 31
Sundar Pichai 1.00 1.00 1.00 27
Thomas Middleditch 1.00 0.97 0.98 31
Tom Cavanagh 0.87 1.00 0.93 20
Ursula Corbero 0.96 0.96 0.96 25
Wentworth Miller 0.93 1.00 0.97 28
Willa Holland 0.98 0.98 0.98 42
William Fichtner 0.98 0.95 0.96 43
alexandra daddario 0.98 0.98 0.98 49
alycia debnam carey 0.95 0.98 0.97 43
amber heard 1.00 0.93 0.96 44
anne hathaway 1.00 0.97 0.99 38
barbara palvin 0.97 0.97 0.97 32
bellamy blake 0.96 0.83 0.89 30
bill gates 1.00 0.97 0.98 30
brie larson 0.95 0.89 0.92 46
chadwick boseman 1.00 1.00 1.00 42
david mazouz 1.00 1.00 1.00 30
drake 1.00 0.92 0.96 13
dua lipa 1.00 0.96 0.98 49
eliza taylor 0.91 0.89 0.90 36
elizabeth olsen 0.96 1.00 0.98 48
elon musk 1.00 1.00 1.00 29
emma watson 0.80 1.00 0.89 45
gal gadot 0.85 1.00 0.92 40
grant gustin 1.00 1.00 1.00 39
jason isaacs 0.98 0.93 0.95 44
jeff bezos 1.00 1.00 1.00 23
kiernan shipka 1.00 0.98 0.99 47
kristen stewart 0.90 0.87 0.88 30
lindsey morgan 1.00 0.92 0.96 26
margot robbie 0.93 0.93 0.93 41
maria pedraza 0.80 1.00 0.89 16
mark zuckerberg 1.00 1.00 1.00 22
miguel herran 1.00 0.96 0.98 23
millie bobby brown 0.89 0.96 0.93 26
rihanna 1.00 1.00 1.00 38
robert downey jr 0.95 0.95 0.95 22
sean pertwee 0.95 0.95 0.95 22
selena gomez 0.84 0.96 0.90 27
shakira 1.00 0.84 0.91 19
tati gabrielle 1.00 0.94 0.97 18
taylor swift 1.00 1.00 1.00 39
tom holland 0.91 1.00 0.95 29
zendaya 1.00 0.96 0.98 27
accuracy 0.96 3231
macro avg 0.96 0.96 0.96 3231
weighted avg 0.97 0.96 0.96 3231
def predict_name(file_name):
# Load the input image
img = read_image(file_name)
# Normalizing pixel values and resize image
img = (img/255.).astype(np.float32)
img = cv2.resize(img, (224, 224))
# Generate the embedding vector for the image
img_embedding = face_descriptor.predict(np.expand_dims(img, axis = 0))[0]
# Apply standard scaler on the vector and reshape
img_embedding_scaled = sc.transform(img_embedding.reshape(1, -1))
# Apply PCA and predict using SVM model
img_pred = svc_model.predict(pca.transform(img_embedding_scaled))
name_pred = classid_to_name[img_pred[0]]
return img_pred[0], name_pred
# Images to predict
# Per problem statement, following image files are provided to predict.
img1 = 'Part 2 - Test Image - Dwayne Johnson4.jpg'
img2 = 'Part 2- Test Image - Benedict Cumberbatch9.jpg'
# pred_images = ['P02-Predict-image-01.jpg', 'P02-Predict-image-02.jpg']
pred_images = [img1, img2]
for image in pred_images:
disp_img = cv2.resize(read_image(image), (224, 224))
img_class, class_nm = predict_name(image)
fig = plt.figure(figsize = (15, 5))
plt.axis('off')
plt.imshow(disp_img)
plt.title(f"Name: {class_nm} \n Class -ID : {img_class}")
plt.show()
# PREDICTION OUTCOME
# Correctly predicted both the test images